diff --git a/packages/block-editor/src/components/colors-gradients/dropdown.js b/packages/block-editor/src/components/colors-gradients/dropdown.js
index 5c76ccb6f9..b13e22b353 100644
--- a/packages/block-editor/src/components/colors-gradients/dropdown.js
+++ b/packages/block-editor/src/components/colors-gradients/dropdown.js
@@ -7,11 +7,11 @@ import classnames from 'classnames';
  * WordPress dependencies
  */
 import {
+	Button,
 	ColorIndicator,
 	Dropdown,
 	FlexItem,
 	__experimentalHStack as HStack,
-	__experimentalItem as Item,
 	__experimentalToolsPanelItem as ToolsPanelItem,
 } from '@wordpress/components';
 
@@ -45,6 +45,9 @@ const WithToolsPanelItem = ( { setting, children, panelId, ...props } ) => {
 			{ ...props }
 			className="block-editor-tools-panel-color-gradient-settings__item"
 			panelId={ panelId }
+			// Pass resetAllFilter if supplied due to rendering via SlotFill
+			// into parent ToolsPanel.
+			resetAllFilter={ setting.resetAllFilter }
 		>
 			{ children }
 		</ToolsPanelItem>
@@ -70,16 +73,16 @@ const renderToggle = ( settings ) => ( { onToggle, isOpen } ) => {
 	const toggleProps = {
 		onClick: onToggle,
 		className: classnames(
-			'block-editor-tools-panel-color-gradient-settings__dropdown',
+			'block-editor-panel-color-gradient-settings__dropdown',
 			{ 'is-open': isOpen }
 		),
 		'aria-expanded': isOpen,
 	};
 
 	return (
-		<Item { ...toggleProps }>
+		<Button { ...toggleProps }>
 			<LabeledColorIndicator colorValue={ colorValue } label={ label } />
-		</Item>
+		</Button>
 	);
 };
 
diff --git a/packages/block-editor/src/components/colors-gradients/panel-color-gradient-settings.js b/packages/block-editor/src/components/colors-gradients/panel-color-gradient-settings.js
index 1cc4b65ef7..797a6c9463 100644
--- a/packages/block-editor/src/components/colors-gradients/panel-color-gradient-settings.js
+++ b/packages/block-editor/src/components/colors-gradients/panel-color-gradient-settings.js
@@ -43,7 +43,7 @@ export const PanelColorGradientSettingsInner = ( {
 	__experimentalIsRenderedInSidebar,
 	enableAlpha,
 } ) => {
-	const panelId = useInstanceId( PanelColorGradientSettingsInner )
+	const panelId = useInstanceId( PanelColorGradientSettingsInner );
 	const { batch } = useRegistry();
 	if (
 		isEmpty( colors ) &&
@@ -90,8 +90,6 @@ export const PanelColorGradientSettingsInner = ( {
 				} );
 			} }
 			panelId={ panelId }
-			hasInnerWrapper={ true }
-			shouldRenderPlaceholderItems={ true } // Required to maintain fills ordering.
 			__experimentalFirstVisibleItemClass="first"
 			__experimentalLastVisibleItemClass="last"
 		>
diff --git a/packages/block-editor/src/components/colors-gradients/style.scss b/packages/block-editor/src/components/colors-gradients/style.scss
index abac92db1b..20c146ee71 100644
--- a/packages/block-editor/src/components/colors-gradients/style.scss
+++ b/packages/block-editor/src/components/colors-gradients/style.scss
@@ -112,3 +112,7 @@
 		}
 	}
 }
+
+.block-editor-panel-color-gradient-settings__dropdown {
+	width: 100%;
+}
diff --git a/packages/block-editor/src/hooks/color-panel.js b/packages/block-editor/src/hooks/color-panel.js
index 34a8645ea7..d0775abbc4 100644
--- a/packages/block-editor/src/hooks/color-panel.js
+++ b/packages/block-editor/src/hooks/color-panel.js
@@ -6,12 +6,11 @@ import { useState, useEffect } from '@wordpress/element';
 /**
  * Internal dependencies
  */
-import PanelColorGradientSettings from '../components/colors-gradients/panel-color-gradient-settings';
+import ColorGradientSettingsDropdown from '../components/colors-gradients/dropdown';
 import ContrastChecker from '../components/contrast-checker';
 import InspectorControls from '../components/inspector-controls';
 import useMultipleOriginColorsAndGradients from '../components/colors-gradients/use-multiple-origin-colors-and-gradients';
 import { __unstableUseBlockRef as useBlockRef } from '../components/block-list/use-block-props/use-block-refs';
-import { __ } from '@wordpress/i18n';
 
 function getComputedStyle( node ) {
 	return node.ownerDocument.defaultView.getComputedStyle( node );
@@ -63,24 +62,24 @@ export default function ColorPanel( {
 	const colorGradientSettings = useMultipleOriginColorsAndGradients();
 
 	return (
-		<InspectorControls>
-			<PanelColorGradientSettings
+		<InspectorControls __experimentalGroup="color">
+			<ColorGradientSettingsDropdown
 				enableAlpha={ enableAlpha }
+				panelId={ clientId }
 				settings={ settings }
+				__experimentalIsItemGroup={ false }
 				__experimentalHasMultipleOrigins
 				__experimentalIsRenderedInSidebar
 				{ ...colorGradientSettings }
-				title={ __( 'Color' ) }
-			>
-				{ enableContrastChecking && (
-					<ContrastChecker
-						backgroundColor={ detectedBackgroundColor }
-						textColor={ detectedColor }
-						enableAlphaChecker={ enableAlpha }
-						linkColor={ detectedLinkColor }
-					/>
-				) }
-			</PanelColorGradientSettings>
+			/>
+			{ enableContrastChecking && (
+				<ContrastChecker
+					backgroundColor={ detectedBackgroundColor }
+					textColor={ detectedColor }
+					enableAlphaChecker={ enableAlpha }
+					linkColor={ detectedLinkColor }
+				/>
+			) }
 		</InspectorControls>
 	);
 }
diff --git a/packages/block-editor/src/hooks/color.js b/packages/block-editor/src/hooks/color.js
index 52ade28f37..3219e01692 100644
--- a/packages/block-editor/src/hooks/color.js
+++ b/packages/block-editor/src/hooks/color.js
@@ -76,6 +76,60 @@ const hasTextColorSupport = ( blockType ) => {
 	return colorSupport && colorSupport.text !== false;
 };
 
+/**
+ * Clears a single color property from a style object.
+ *
+ * @param {Array}  path  Path to color property to clear within styles object.
+ * @param {Object} style Block attributes style object.
+ * @return {Object} Styles with the color property omitted.
+ */
+const clearColorFromStyles = ( path, style ) =>
+	cleanEmptyObject( immutableSet( style, path, undefined ) );
+
+/**
+ * Clears text color related properties from supplied attributes.
+ *
+ * @param {Object} attributes Block attributes.
+ * @return {Object} Update block attributes with text color properties omitted.
+ */
+const resetAllTextFilter = ( attributes ) => ( {
+	textColor: undefined,
+	style: clearColorFromStyles( [ 'color', 'text' ], attributes.style ),
+} );
+
+/**
+ * Clears link color related properties from supplied attributes.
+ *
+ * @param {Object} attributes Block attributes.
+ * @return {Object} Update block attributes with link color properties omitted.
+ */
+const resetAllLinkFilter = ( attributes ) => ( {
+	style: clearColorFromStyles(
+		[ 'elements', 'link', 'color', 'text' ],
+		attributes.style
+	),
+} );
+
+/**
+ * Clears all background color related properties including gradients from
+ * supplied block attributes.
+ *
+ * @param {Object} attributes Block attributes.
+ * @return {Object} Block attributes with background and gradient omitted.
+ */
+const clearBackgroundAndGradient = ( attributes ) => ( {
+	backgroundColor: undefined,
+	gradient: undefined,
+	style: {
+		...attributes.style,
+		color: {
+			...attributes.style?.color,
+			background: undefined,
+			gradient: undefined,
+		},
+	},
+} );
+
 /**
  * Filters registered block settings, extending attributes to include
  * `backgroundColor` and `textColor` attribute.
@@ -414,6 +468,7 @@ export function ColorEdit( props ) {
 									style?.color?.text
 								).color,
 								isShownByDefault: defaultColorControls?.text,
+								resetAllFilter: resetAllTextFilter,
 							},
 					  ]
 					: [] ),
@@ -435,6 +490,7 @@ export function ColorEdit( props ) {
 									: undefined,
 								isShownByDefault:
 									defaultColorControls?.background,
+								resetAllFilter: clearBackgroundAndGradient,
 							},
 					  ]
 					: [] ),
@@ -450,6 +506,7 @@ export function ColorEdit( props ) {
 								clearable: !! style?.elements?.link?.color
 									?.text,
 								isShownByDefault: defaultColorControls?.link,
+								resetAllFilter: resetAllLinkFilter,
 							},
 					  ]
 					: [] ),
diff --git a/packages/block-editor/src/hooks/color.scss b/packages/block-editor/src/hooks/color.scss
index 2b79f03a50..f9b7898f94 100644
--- a/packages/block-editor/src/hooks/color.scss
+++ b/packages/block-editor/src/hooks/color.scss
@@ -20,4 +20,13 @@
 			row-gap: 0;
 		}
 	}
+
+	/**
+	 * After converting PanelColorGradientSettings to render as a ToolsPanel
+	 * we need to remove the top margin when wrapping inner content due to
+	 * rendering via SlotFills.
+	 */
+	.block-editor-tools-panel-color-gradient-settings__item.first {
+		margin-top: 0;
+	}
 }
diff --git a/packages/block-library/src/social-links/edit.js b/packages/block-library/src/social-links/edit.js
index ed148909ad..f06f2ff42b 100644
--- a/packages/block-library/src/social-links/edit.js
+++ b/packages/block-library/src/social-links/edit.js
@@ -215,16 +215,17 @@ export function SocialLinksEdit( props ) {
 					__experimentalIsRenderedInSidebar
 					title={ __( 'Color' ) }
 					colorSettings={ colorSettings }
-				/>
-				{ ! logosOnly && (
-					<ContrastChecker
-						{ ...{
-							textColor: iconColorValue,
-							backgroundColor: iconBackgroundColorValue,
-						} }
-						isLargeText={ false }
-					/>
-				) }
+				>
+					{ ! logosOnly && (
+						<ContrastChecker
+							{ ...{
+								textColor: iconColorValue,
+								backgroundColor: iconBackgroundColorValue,
+							} }
+							isLargeText={ false }
+						/>
+					) }
+				</PanelColorSettings>
 			</InspectorControls>
 			<ul { ...innerBlocksProps } />
 		</Fragment>
diff --git a/packages/components/src/tools-panel/styles.ts b/packages/components/src/tools-panel/styles.ts
index 715707a100..6dec56f1b5 100644
--- a/packages/components/src/tools-panel/styles.ts
+++ b/packages/components/src/tools-panel/styles.ts
@@ -54,8 +54,7 @@ export const ToolsPanelWithInnerWrapper = ( columns: number ) => {
 };
 
 export const ToolsPanelHiddenInnerWrapper = css`
-	/* Required to meet specificity requirements */
-	&&& > div:not( :first-of-type ) {
+	> div:not( :first-of-type ) {
 		display: none;
 	}
 `;